# -*- coding: utf-8 -*-
# @Author : Ace
# @File : 淘宝GUI线程队列商品信息采集.py
# @Software: PyCharm

import tkinter as tk
import tkinter.messagebox as msgbox
import os, xlwt, xlrd
from xlutils.copy import copy
from threading import Thread
from queue import Queue
from requests_html import HTMLSession
import time, hashlib, json
from urllib.parse import quote
session = HTMLSession()

COOKIE = 'cna=74aWGrOntgkCASSOntSec4mr; xlly_s=1; sgcookie=E100eggiaco%2BxJMq0bb64%2BBJ7bEMR7aBq2r8tKa0laspj2eqYrk6QiKVrCmvpNNor6RyUjE2fOhu63GreNG1vTOe8%2BwL22P1BzO2LFSJNW0q0Sk%3D; uc3=lg2=Vq8l%2BKCLz3%2F65A%3D%3D&id2=UUpgRKrwhWXVFFEXKA%3D%3D&vt3=F8dCvUFihkJHBGn3kz8%3D&nk2=F5RDJXYb5z2oPIhy; lgc=tb6935546184; uc4=id4=0%40U2gqy1tzPVviToJYCTjrH9pM0UevAC3X&nk4=0%40FY4I5qGhDEiUK1vhD8qCSH4HJk9JgxY%3D; tracknick=tb6935546184; _cc_=U%2BGCWk%2F7og%3D%3D; thw=cn; mt=ci=-1_0; enc=AWUg85ofcsD%2B%2Ffh9sEkm2A8aZ7FcmZ0qjeNBQUtx49MHzd6IlihlYz1hVPfbk4fKmUJM4%2FWG1FpLcx%2BktSxOpQnTSoWAq9iI6IX00ddhACg%3D; t=455ddf181a2dd275e346e7be4d4a896c; _m_h5_tk=194c15db1f9b5abf2cdbf3301fc30f28_1646841365905; _m_h5_tk_enc=2254d1ca40efac8b3ef8f04b0127135e; _tb_token_=f3d8b777b06b5; uc1=cookie14=UoewBjJbOlKUfg%3D%3D; tfstk=cc8RBV03BxDoL6fv_3nDdHE4C7Gcw4PRK76LJjXUBSIAGt1m1FYTDsi5TOW-M; l=eBg--JknLaEjJUuwBOfwourza77OSIRAguPzaNbMiOCPO_fp52k5W6mOiZT9C3GVhsHHR3lrOrBaBeYBqn4jPGKnNSVsr4Dmn; isg=BBcXO8fPOUYeXL3KX1IxX7txpothXOu-vjswS2lEM-ZNmDfacSx7DtW--jiGcMM2'
H5_TOKEN = '98a1abc671c848e30df7ff5a80ddaee1'


class TBSpider(object):

    def __init__(self):
        """
        爬虫原理的第一步：准备数据
        """
        # 起始的地址
        self.start_url = 'https://h5api.m.taobao.com/h5/mtop.alimama.union.xt.en.api.entry/1.0/'
        # 创建队列对象
        self.que = Queue()
        # 定义检索词列表
        # self.search_str_list = ['情人节', '表白礼物', '化妆品', '衣服']
        # 定义通用的headers(请求头)
        self.headers = {
            'cookie': COOKIE,
            'referer': 'https://uland.taobao.com/',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
        }
        """=========================================分割线=========================================="""
        """定义可视化窗口，并设置窗口和主题大小布局"""
        self.window = tk.Tk()
        self.window.title('某宝数据采集')
        self.window.geometry('800x600')

        """创建label_user按钮，与说明书"""
        self.label_user = tk.Label(self.window, text='请输入检索词：', font=('Arial', 12), width=30, height=2)
        self.label_user.pack()
        """创建label_user关联输入"""
        self.entry_user = tk.Entry(self.window, show=None, font=('Arial', 14))
        self.entry_user.pack(after=self.label_user)

        """创建Text富文本框，用于按钮操作结果的展示"""
        # 定义富文本框滑动条
        scroll = tk.Scrollbar()
        # 放到窗口的右侧, 填充Y竖直方向
        scroll.pack(side=tk.RIGHT, fill=tk.Y)
        self.text1 = tk.Text(self.window, font=('Arial', 12), width=85, height=20)
        self.text1.pack()
        # 两个控件关联
        scroll.config(command=self.text1.yview)
        self.text1.config(yscrollcommand=scroll.set)

        """定义按钮1，绑定触发事件方法"""
        """即登录按钮，当点击时将执行parse_hit_click_1方法。在真实使用场景中"""
        """parse_hit_click_1中可替换为自己写的真正登录函数。这里仅为示例"""
        self.button_1 = tk.Button(self.window, text='运行，采集', font=('Arial', 12), width=10, height=1,
                                  command=self.thread_run)
        self.button_1.pack(before=self.text1)

        """定义按钮2，绑定触发事件方法"""
        self.button_2 = tk.Button(self.window, text='清除', font=('Arial', 12), width=10, height=1,
                                  command=self.parse_hit_click_2)
        self.button_2.pack(anchor="e")

    def thread_run(self):
        """关联启动方法"""
        # 获取用户的输入
        self.user_input_str = self.entry_user.get()
        Thread(target=self.run, args=(self.user_input_str,)).start()

    def parse_join_job(self, tag):
        """
        负责往队列容器中添加任务
        :return:
        """
        for page in range(1, 10):
            # 生成时间戳
            temp = str(int(time.time() * 1000))
            # GET请求查询关键参数
            data = {"pNum": page, "pSize": "60", "refpid": "mm_26632258_3504122_32538762",
                    "variableMap": "{\"q\":\"" + tag + "\",\"navigator\":false,\"clk1\":\"2611f14a9b25c784be96af7d37dd2b57\",\"union_lens\":\"recoveryid:201_11.21.30.45_5549665_1629979272500;prepvid:201_11.21.30.45_5549665_1629979272500\",\"recoveryId\":\"201_11.11.144.128_3497176_1629979569186\"}",
                    "qieId": "36308", "spm": "a2e0b.20350158.31919782",
                    "app_pvid": "201_11.11.144.128_3497176_1629979569186",
                    "ctm": "spm-url:a2e0b.20350158.31919782.1;page_url:https%3A%2F%2Fuland.taobao.com%2Fsem%2Ftbsearch%3Fspm%3Da2e0b.20350158.31919782.1%26refpid%3Dmm_26632258_3504122_32538762%26keyword%3D%25E5%2589%2583%25E9%25A1%25BB%25E5%2588%2580%26clk1%3D2611f14a9b25c784be96af7d37dd2b57%26upsId%3D2611f14a9b25c784be96af7d37dd2b57%26pid%3Dmm_26632258_3504122_32538762%26union_lens%3Drecoveryid%253A201_11.21.30.45_5549665_1629979272500%253Bprepvid%253A201_11.21.30.45_5549665_1629979272500%26pnum%3D2"}
            # 拼接完整的请求地址
            start_url = f'https://h5api.m.taobao.com/h5/mtop.alimama.union.xt.en.api.entry/1.0/?jsv=2.5.1&appKey=12574478&t={temp}&sign={self.parse_sign(temp, data)}&api=mtop.alimama.union.xt.en.api.entry&v=1.0&AntiCreep=true&timeout=20000&AntiFlood=true&type=jsonp&dataType=jsonp&callback=mtopjsonp3&data='
            # 开始拼接完整的请求地址
            result_url = start_url + quote(str(data))
            """往队列容器中添加任务"""
            self.que.put(result_url)
        self.que.join()

    def parse_get_que_job(self):
        """
        从容器队列对象中，获取任务
        :return:
        """
        while True:
            # 获取任务
            url = self.que.get()
            # 发送请求，获取响应
            response = session.get(url, headers=self.headers).content.decode()
            """类中，方法之间的调用"""
            self.parse_response_data(response)
            self.que.task_done()

    def parse_response_data(self, response):
        """
        解析response响应，提取数据
        :param response: 响应对象
        :return:
        """
        # 数据过滤
        json_data = response[12:-1]
        print(json_data)
        # print(json_data)
        try:
            # 转换数据类型
            dict_data_list = json.loads(json_data)['data']['recommend']['resultList']
            # 提取商品数据大列表
            for dict_data in dict_data_list:
                # floorId
                floorId = dict_data['floorId']
                # itemName 商品名称
                itemName = dict_data['itemName']
                # itemId
                itemId = dict_data['itemId']
                # shopTitle 商品店铺名称
                shopTitle = dict_data['shopTitle']
                # 店铺服务评分
                dsrScore = dict_data['dsrScore']
                # 发货地址
                provcity = dict_data['provcity']
                # 商品详情页
                info_url = dict_data['url']
                # 商品最低价格
                promotionPrice = dict_data['promotionPrice']
                """构造保存数据格式大字典"""
                dict_list_data = {
                    f'{self.user_input_str}': [floorId, itemName, promotionPrice, provcity, itemId, shopTitle, dsrScore, info_url]
                }
                self.SaveExcels(dict_list_data)
                printinfo = f"{itemName}--------数据采集完成----------logging！！！"
                self.text1.insert("insert", printinfo + "\n")
                # gui界面滑动条自动下拉
                self.text1.see('insert')
        except:
            msgbox.showerror(title='错误', message='无效，请重新输入！')
            self.parse_hit_click_2()

    def SaveExcels(self, data):
        """
        使用前，请先阅读代码
        :param data: 需要保存的data字典(有格式要求)
        :return:
        格式要求:
            data = {
            '基本详情': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']
        }
        注意：这个模板程序会将data的key值作为excel的表名来判断，一样才会保存
        """
        # 获取表的名称
        sheet_name = [i for i in data.keys()][0]
        # 创建保存excel表格的文件夹
        # os.getcwd() 获取当前文件路径
        os_mkdir_path = os.getcwd() + '/数据/'
        # 判断这个路径是否存在，不存在就创建
        if not os.path.exists(os_mkdir_path):
            os.mkdir(os_mkdir_path)
        # 判断excel表格是否存在           工作簿文件名称
        os_excel_path = os_mkdir_path + '数据.xls'
        if not os.path.exists(os_excel_path):
            # 不存在，创建工作簿(也就是创建excel表格)
            workbook = xlwt.Workbook(encoding='utf-8')
            """工作簿中创建新的sheet表"""  # 设置表名
            worksheet1 = workbook.add_sheet(sheet_name, cell_overwrite_ok=True)
            """设置sheet表的表头"""
            sheet1_headers = ('商品的id', '商品名称', '商品价格', '发货地址', '店铺的id', '店铺的名称', '店铺的服务评分', '商品详情页地址')
            # 将表头写入工作簿
            for header_num in range(0, len(sheet1_headers)):
                # 设置表格长度
                worksheet1.col(header_num).width = 2560 * 3
                # 写入表头        行,    列,           内容
                worksheet1.write(0, header_num, sheet1_headers[header_num])
            # 循环结束，代表表头写入完成，保存工作簿
            workbook.save(os_excel_path)
        """=============================已有工作簿添加新表==============================================="""
        # 打开工作薄
        workbook = xlrd.open_workbook(os_excel_path)
        # 获取工作薄中所有表的名称
        sheets_list = workbook.sheet_names()
        # 如果表名称：字典的key值不在工作簿的表名列表中
        if sheet_name not in sheets_list:
            # 复制先有工作簿对象
            work = copy(workbook)
            # 通过复制过来的工作簿对象，创建新表  -- 保留原有表结构
            sh = work.add_sheet(sheet_name)
            # 给新表设置表头
            excel_headers_tuple = ('商品的id', '商品名称', '商品价格', '发货地址', '店铺的id', '店铺的名称', '店铺的服务评分', '商品详情页地址')
            for head_num in range(0, len(excel_headers_tuple)):
                sh.col(head_num).width = 2560 * 3
                #               行，列，  内容，            样式
                sh.write(0, head_num, excel_headers_tuple[head_num])
            work.save(os_excel_path)
        """========================================================================================="""
        # 判断工作簿是否存在
        if os.path.exists(os_excel_path):
            # 打开工作簿
            workbook = xlrd.open_workbook(os_excel_path)
            # 获取工作薄中所有表的个数
            sheets = workbook.sheet_names()
            for i in range(len(sheets)):
                for name in data.keys():
                    worksheet = workbook.sheet_by_name(sheets[i])
                    # 获取工作薄中所有表中的表名与数据名对比
                    if worksheet.name == name:
                        # 获取表中已存在的行数
                        rows_old = worksheet.nrows
                        # 将xlrd对象拷贝转化为xlwt对象
                        new_workbook = copy(workbook)
                        # 获取转化后的工作薄中的第i张表
                        new_worksheet = new_workbook.get_sheet(i)
                        for num in range(0, len(data[name])):
                            new_worksheet.write(rows_old, num, data[name][num])
                        new_workbook.save(os_excel_path)

    def parse_sign(self, temp, data):
        """
        解析sign请求参数
        h(d.token + "&" + i + "&" + g + "&" + c.data)
        :return:
        """
        # 有效token值，在cookie中可以找到
        h5_tk = H5_TOKEN
        # 加密算法还原拼接语法
        result = h5_tk + "&" + temp + "&" + '12574478' + "&" + f"{data}"
        # md5算法还原
        sign = hashlib.md5(result.encode(encoding='utf-8')).hexdigest()
        return sign

    def run(self, tag):
        """
        运行爬虫
        :return:
        """
        # 创建线程对象t1:负责往容器队列中添加任务
        t1 = Thread(target=self.parse_join_job, args=(tag,))
        # 创建线程对象t2:负责从容器队列中获取任务
        t2 = Thread(target=self.parse_get_que_job)
        # 启动t1线程
        t1.start()
        # 将t2线程设置为守护线程：
        # 让t2线程随着t1线程结束而结束
        t2.daemon = True
        t2.start()

    def parse_hit_click_2(self):
        """定义触发事件2，删除文本框中内容"""
        self.entry_user.delete(0, "end")
        # self.entry_passwd.delete(0, "end")
        self.text1.delete("1.0", "end")

    def center(self):
        """创建窗口居中函数方法"""
        ws = self.window.winfo_screenwidth()
        hs = self.window.winfo_screenheight()
        x = int((ws / 2) - (800 / 2))
        y = int((hs / 2) - (600 / 2))
        self.window.geometry('{}x{}+{}+{}'.format(800, 600, x, y))

    def run_loop(self):
        """禁止修改窗体大小规格"""
        self.window.resizable(False, False)
        """窗口居中"""
        self.center()
        """窗口维持--持久化"""
        self.window.mainloop()


if __name__ == '__main__':
    t = TBSpider()
    t.run_loop()